<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Reranker Framework (ReFr): stream-tokenizer.C Source File</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Reranker Framework (ReFr)
   
   </div>
   <div id="projectbrief">Reranking framework for structure prediction and discriminative language modeling</div>
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="namespaces.html"><span>Namespaces</span></a></li>
      <li><a href="annotated.html"><span>Classes</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="headertitle">
<div class="title">stream-tokenizer.C</div>  </div>
</div><!--header-->
<div class="contents">
<a href="stream-tokenizer_8_c.html">Go to the documentation of this file.</a><div class="fragment"><pre class="fragment"><a name="l00001"></a>00001 <span class="comment">// Copyright 2012, Google Inc.</span>
<a name="l00002"></a>00002 <span class="comment">// All rights reserved.</span>
<a name="l00003"></a>00003 <span class="comment">//</span>
<a name="l00004"></a>00004 <span class="comment">// Redistribution and use in source and binary forms, with or without</span>
<a name="l00005"></a>00005 <span class="comment">// modification, are permitted provided that the following conditions are</span>
<a name="l00006"></a>00006 <span class="comment">// met:</span>
<a name="l00007"></a>00007 <span class="comment">//</span>
<a name="l00008"></a>00008 <span class="comment">//   * Redistributions of source code must retain the above copyright</span>
<a name="l00009"></a>00009 <span class="comment">//     notice, this list of conditions and the following disclaimer.</span>
<a name="l00010"></a>00010 <span class="comment">//   * Redistributions in binary form must reproduce the above</span>
<a name="l00011"></a>00011 <span class="comment">//     copyright notice, this list of conditions and the following disclaimer</span>
<a name="l00012"></a>00012 <span class="comment">//     in the documentation and/or other materials provided with the</span>
<a name="l00013"></a>00013 <span class="comment">//     distribution.</span>
<a name="l00014"></a>00014 <span class="comment">//   * Neither the name of Google Inc. nor the names of its</span>
<a name="l00015"></a>00015 <span class="comment">//     contributors may be used to endorse or promote products derived from</span>
<a name="l00016"></a>00016 <span class="comment">//     this software without specific prior written permission.</span>
<a name="l00017"></a>00017 <span class="comment">//</span>
<a name="l00018"></a>00018 <span class="comment">// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS</span>
<a name="l00019"></a>00019 <span class="comment">// &quot;AS IS&quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT</span>
<a name="l00020"></a>00020 <span class="comment">// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR</span>
<a name="l00021"></a>00021 <span class="comment">// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT</span>
<a name="l00022"></a>00022 <span class="comment">// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span>
<a name="l00023"></a>00023 <span class="comment">// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT</span>
<a name="l00024"></a>00024 <span class="comment">// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,</span>
<a name="l00025"></a>00025 <span class="comment">// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY</span>
<a name="l00026"></a>00026 <span class="comment">// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT</span>
<a name="l00027"></a>00027 <span class="comment">// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE</span>
<a name="l00028"></a>00028 <span class="comment">// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span>
<a name="l00029"></a>00029 <span class="comment">// -----------------------------------------------------------------------------</span>
<a name="l00030"></a>00030 <span class="comment">//</span>
<a name="l00031"></a>00031 <span class="comment">//</span>
<a name="l00036"></a>00036 <span class="comment"></span>
<a name="l00037"></a>00037 <span class="preprocessor">#include &lt;ctype.h&gt;</span>
<a name="l00038"></a>00038 <span class="preprocessor">#include &lt;stdexcept&gt;</span>
<a name="l00039"></a>00039 
<a name="l00040"></a>00040 <span class="preprocessor">#include &quot;<a class="code" href="stream-tokenizer_8_h.html" title="Provides the StreamTokenizer  class.">stream-tokenizer.H</a>&quot;</span>
<a name="l00041"></a>00041 
<a name="l00042"></a>00042 <span class="keyword">namespace </span>reranker {
<a name="l00043"></a>00043 
<a name="l00044"></a>00044 <span class="keywordtype">void</span>
<a name="l00045"></a>00045 StreamTokenizer::ConsumeChar(<span class="keywordtype">char</span> c) {
<a name="l00046"></a>00046   oss_ &lt;&lt; c;
<a name="l00047"></a>00047   ++num_read_;
<a name="l00048"></a>00048   <span class="keywordflow">if</span> (c == <span class="charliteral">&#39;\n&#39;</span>) {
<a name="l00049"></a>00049     ++line_number_;
<a name="l00050"></a>00050   }
<a name="l00051"></a>00051 }
<a name="l00052"></a>00052 
<a name="l00053"></a>00053 <span class="keywordtype">bool</span>
<a name="l00054"></a>00054 StreamTokenizer::ReadChar(<span class="keywordtype">char</span> *c) {
<a name="l00055"></a>00055   (*c) = is_.get();
<a name="l00056"></a>00056   <span class="keywordflow">if</span> (!is_.good()) {
<a name="l00057"></a>00057     has_next_ = <span class="keyword">false</span>;
<a name="l00058"></a>00058     <span class="keywordflow">return</span> <span class="keyword">false</span>;
<a name="l00059"></a>00059   } <span class="keywordflow">else</span> {
<a name="l00060"></a>00060     ConsumeChar(*c);
<a name="l00061"></a>00061     <span class="keywordflow">return</span> <span class="keyword">true</span>;
<a name="l00062"></a>00062   }
<a name="l00063"></a>00063 }
<a name="l00064"></a>00064 
<a name="l00065"></a>00065 <span class="keywordtype">void</span>
<a name="l00066"></a>00066 StreamTokenizer::GetNext() {
<a name="l00067"></a>00067   <span class="keywordflow">if</span> (!is_.good()) {
<a name="l00068"></a>00068     has_next_ = <span class="keyword">false</span>;
<a name="l00069"></a>00069     <span class="keywordflow">return</span>;
<a name="l00070"></a>00070   }
<a name="l00071"></a>00071   <span class="comment">// Get first character of next token.</span>
<a name="l00072"></a>00072   <span class="keywordtype">char</span> c;
<a name="l00073"></a>00073   <span class="keywordtype">bool</span> is_whitespace = <span class="keyword">true</span>;
<a name="l00074"></a>00074   <span class="keywordflow">while</span> (is_whitespace) {
<a name="l00075"></a>00075     <span class="keywordflow">if</span> (!ReadChar(&amp;c)) {
<a name="l00076"></a>00076       <span class="keywordflow">return</span>;
<a name="l00077"></a>00077     }
<a name="l00078"></a>00078     is_whitespace = isspace(c);
<a name="l00079"></a>00079 
<a name="l00080"></a>00080     <span class="comment">// If we find a comment character, then read to the end of the line.</span>
<a name="l00081"></a>00081     <span class="keywordflow">if</span> (!is_whitespace &amp;&amp; c == <span class="charliteral">&#39;/&#39;</span>) {
<a name="l00082"></a>00082       <span class="keywordflow">while</span> (c != <span class="charliteral">&#39;\n&#39;</span>) {
<a name="l00083"></a>00083         <span class="keywordflow">if</span> (!ReadChar(&amp;c)) {
<a name="l00084"></a>00084           <span class="keywordflow">return</span>;
<a name="l00085"></a>00085         }
<a name="l00086"></a>00086       }
<a name="l00087"></a>00087       is_whitespace = <span class="keyword">true</span>;
<a name="l00088"></a>00088     }
<a name="l00089"></a>00089   }
<a name="l00090"></a>00090 
<a name="l00091"></a>00091   has_next_ = <span class="keyword">true</span>;
<a name="l00092"></a>00092   next_tok_start_ = num_read_ - 1;
<a name="l00093"></a>00093   <span class="keywordtype">bool</span> next_tok_complete = <span class="keyword">false</span>;
<a name="l00094"></a>00094   next_tok_.clear();
<a name="l00095"></a>00095   <span class="keywordflow">if</span> (ReservedChar(c)) {
<a name="l00096"></a>00096     next_tok_ += c;
<a name="l00097"></a>00097     next_tok_complete = <span class="keyword">true</span>;
<a name="l00098"></a>00098     next_tok_type_ = <a class="code" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daaf92bc177d9920c389a149fa718c9ff3a">RESERVED_CHAR</a>;
<a name="l00099"></a>00099   } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (c == <span class="charliteral">&#39;&quot;&#39;</span>) {
<a name="l00100"></a>00100     <span class="comment">// We&#39;ve got a string literal, so keep reading characters,</span>
<a name="l00101"></a>00101     <span class="comment">// until hitting a non-escaped double quote.</span>
<a name="l00102"></a>00102     streampos string_literal_start_pos = num_read_ - 1;
<a name="l00103"></a>00103     <span class="keywordtype">bool</span> found_closing_quote = <span class="keyword">false</span>;
<a name="l00104"></a>00104     <span class="keywordflow">while</span> (is_.good()) {
<a name="l00105"></a>00105       <span class="keywordtype">bool</span> success = ReadChar(&amp;c);
<a name="l00106"></a>00106       <span class="keywordflow">if</span> (success) {
<a name="l00107"></a>00107         <span class="keywordflow">if</span> (c == <span class="charliteral">&#39;&quot;&#39;</span>) {
<a name="l00108"></a>00108           found_closing_quote = <span class="keyword">true</span>;
<a name="l00109"></a>00109           <span class="keywordflow">break</span>;
<a name="l00110"></a>00110         } <span class="keywordflow">else</span> <span class="keywordflow">if</span> (c == <span class="charliteral">&#39;\\&#39;</span>) {
<a name="l00111"></a>00111           success = ReadChar(&amp;c);
<a name="l00112"></a>00112         }
<a name="l00113"></a>00113       }
<a name="l00114"></a>00114       <span class="keywordflow">if</span> (success) {
<a name="l00115"></a>00115         next_tok_ += c;
<a name="l00116"></a>00116       }
<a name="l00117"></a>00117     }
<a name="l00118"></a>00118     <span class="keywordflow">if</span> (!found_closing_quote) {
<a name="l00119"></a>00119       cerr &lt;&lt; <span class="stringliteral">&quot;reranker::StreamTokenizer: error: could not find closing &quot;</span>
<a name="l00120"></a>00120            &lt;&lt; <span class="stringliteral">&quot;double quote for string literal beginning at stream index &quot;</span>
<a name="l00121"></a>00121            &lt;&lt; string_literal_start_pos
<a name="l00122"></a>00122            &lt;&lt; <span class="stringliteral">&quot;; partial string literal read: \&quot;&quot;</span>
<a name="l00123"></a>00123            &lt;&lt; next_tok_ &lt;&lt; endl;
<a name="l00124"></a>00124       <span class="keywordflow">throw</span> std::runtime_error(<span class="stringliteral">&quot;unclosed string literal&quot;</span>);
<a name="l00125"></a>00125     }
<a name="l00126"></a>00126     next_tok_complete = <span class="keyword">true</span>;
<a name="l00127"></a>00127     next_tok_type_ = <a class="code" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa6719f11fc4a17c542500088211a08df3">STRING</a>;
<a name="l00128"></a>00128   } <span class="keywordflow">else</span> {
<a name="l00129"></a>00129     <span class="comment">// This is a number, a reserved word or C++ identifier token, so</span>
<a name="l00130"></a>00130     <span class="comment">// add first character; the remainder of the token will be handled</span>
<a name="l00131"></a>00131     <span class="comment">// after this switch statement.</span>
<a name="l00132"></a>00132     next_tok_ += c;
<a name="l00133"></a>00133     next_tok_type_ = (c == <span class="charliteral">&#39;-&#39;</span> || (c &gt;= <span class="charliteral">&#39;0&#39;</span> &amp;&amp; c &lt;= <span class="charliteral">&#39;9&#39;</span>)) ? <a class="code" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa033acb728cba240ddf92404a8e709b79">NUMBER</a> : <a class="code" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa6f3f909fb7f4fdc76c355f131a0cd392">IDENTIFIER</a>;
<a name="l00134"></a>00134   }
<a name="l00135"></a>00135   <span class="keywordflow">if</span> (!next_tok_complete) {
<a name="l00136"></a>00136     <span class="comment">// The current token is a number, a reserved word or C++</span>
<a name="l00137"></a>00137     <span class="comment">// identifier, so we keep reading characters until hitting a</span>
<a name="l00138"></a>00138     <span class="comment">// &quot;reserved character&quot;, a whitespace character or EOF.</span>
<a name="l00139"></a>00139     <span class="keywordtype">bool</span> done = <span class="keyword">false</span>;
<a name="l00140"></a>00140     <span class="keywordflow">while</span> (!done &amp;&amp; is_.good()) {
<a name="l00141"></a>00141       <span class="comment">// We don&#39;t call ReadChar here because we might need to put the</span>
<a name="l00142"></a>00142       <span class="comment">// character back into the underlying stream if it tells us that</span>
<a name="l00143"></a>00143       <span class="comment">// the current token has ended (i.e., if it&#39;s a reserved</span>
<a name="l00144"></a>00144       <span class="comment">// character, a double quote or a whitespace character).</span>
<a name="l00145"></a>00145       c = is_.get();
<a name="l00146"></a>00146       <span class="keywordflow">if</span> (is_.good()) {
<a name="l00147"></a>00147         <span class="keywordflow">if</span> (ReservedChar(c) || c == <span class="charliteral">&#39;&quot;&#39;</span> || isspace(c)) {
<a name="l00148"></a>00148           is_.putback(c);
<a name="l00149"></a>00149           <span class="comment">// Now that we&#39;ve finished reading something that is not a</span>
<a name="l00150"></a>00150           <span class="comment">// string literal, change its type to be RESERVED_WORD if it</span>
<a name="l00151"></a>00151           <span class="comment">// exactly matches something in the set of reserved words.</span>
<a name="l00152"></a>00152           <span class="keywordflow">if</span> (reserved_words_.count(next_tok_) != 0) {
<a name="l00153"></a>00153             next_tok_type_ = <a class="code" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa900de35c73023cf15ec3954852da6809">RESERVED_WORD</a>;
<a name="l00154"></a>00154           }
<a name="l00155"></a>00155           done = <span class="keyword">true</span>;
<a name="l00156"></a>00156         } <span class="keywordflow">else</span> {
<a name="l00157"></a>00157           ConsumeChar(c);
<a name="l00158"></a>00158           next_tok_ += c;
<a name="l00159"></a>00159         }
<a name="l00160"></a>00160       } <span class="keywordflow">else</span> {
<a name="l00161"></a>00161         has_next_ = <span class="keyword">false</span>;
<a name="l00162"></a>00162       }
<a name="l00163"></a>00163     }
<a name="l00164"></a>00164   }
<a name="l00165"></a>00165 }
<a name="l00166"></a>00166 
<a name="l00167"></a>00167 }  <span class="comment">// namespace reranker</span>
</pre></div></div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Tue Apr 9 2013 11:56:29 for Reranker Framework (ReFr) by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
